import json

import scrapy
from scrapy import Request

# 如果有多个爬虫，可以在这里写多个class，注意爬虫名称

from spider.item_market_industryoverview import SpiderItem, ChannelItem, SubChannelItem


class MarketIndustryoverviewSpider(scrapy.Spider):
    name = 'market_industryoverview'

    def start_requests(self):
        for url in ['https://api.bilibili.com/x/web-interface/web/channel/category/list']:
            yield Request(url, meta={}, dont_filter=True)

    def parse(self, response):
        content = json.loads(response.body_as_unicode())
        categories = content['data']['categories']
        for d in categories:
            item = ChannelItem()
            item['id'] = d['id']
            item['name'] = d['name']
            item['count'] = d['channel_count']
            yield item

class MarketIndustryoverviewSpider2(scrapy.Spider):
    name = 'market_industryoverview2'

    def start_requests(self):
        for url in ['https://api.bilibili.com/x/web-interface/web/channel/category/list']:
            yield Request(url, meta={}, dont_filter=True)

    def parse(self, response):
        content = json.loads(response.text)
        categories = content['data']['categories']
        for d in categories:
            if d['id'] == 9:
                url = 'https://api.bilibili.com/x/web-interface/web/channel/category/channel_arc/list?id={}&offset='.format(d['id'])
                yield Request(url, meta={'id':d['id'], 'archive_channels':[]}, callback=self.parse2)

    def parse2(self, response):
        print(response)
        id = response.meta['id']
        archive_channels = response.meta['archive_channels']

        content = json.loads(response.text)
        data = content['data']
        has_more = data['has_more']
        offset = data['offset']
        archive_channels += data['archive_channels']

        if has_more:
            url = 'https://api.bilibili.com/x/web-interface/web/channel/category/channel_arc/list?id={}&offset={}'.format(id, offset)
            yield Request(url, meta={'id':id, 'archive_channels':archive_channels}, callback=self.parse2)
        else:
            for d in archive_channels:
                item = SubChannelItem()
                item['pid'] = id
                item['id'] = d['id']
                item['name'] = d['name']
                item['covor'] = d.get('covor','')
                item['subscribed_count'] = d['subscribed_count']
                item['archive_count'] = d['archive_count']
                yield item

